package com.oj.util;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.gargoylesoftware.htmlunit.*;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.oj.entity.OtherOJ;
import com.oj.entity.RecentContest;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.boot.system.ApplicationHome;

import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.logging.Level;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Slf4j
public class OJCollector {
    public static Document getPageWaitJS (String url) throws IOException {
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        try {
            webClient.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3");
            webClient.setCssErrorHandler(new SilentCssErrorHandler());
            webClient.getCookieManager().setCookiesEnabled(true);
            webClient.getOptions().setUseInsecureSSL(true);
            webClient.getOptions().setJavaScriptEnabled(true);
            webClient.getOptions().setCssEnabled(false);
            webClient.getOptions().setActiveXNative(false);
            webClient.getOptions().setRedirectEnabled(true);
            webClient.getOptions().setThrowExceptionOnScriptError(false);
            webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
            webClient.getOptions().setTimeout(30000); // 设置连接超时时间 ，这里是10S。如果为0，则无限期等待
            webClient.setJavaScriptTimeout(10000);
            webClient.waitForBackgroundJavaScriptStartingBefore(10000);//设置等待js响应时间
            webClient.getOptions().setAppletEnabled(true);//启用小程序
            webClient.getOptions().setGeolocationEnabled(true);//启用定位
            HtmlPage page = webClient.getPage(url);
            webClient.waitForBackgroundJavaScript(10000);
            webClient.setAjaxController(new NicelyResynchronizingAjaxController());
            String pageXml = page.asXml(); //以xml的形式获取响应文本
            //log.error(page.asNormalizedText());
            webClient.close();
            return Jsoup.parse(pageXml);
        } catch (Exception e) {
            e.printStackTrace();
            webClient.close();
            return null;
        }finally {
            webClient.close();
        }
    }
    public static List<RecentContest> CollectHduContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Document document = Jsoup.connect("http://acm.hdu.edu.cn/contests/contest_list.php").get();
            Elements table = document.select("table.table_text");
            Elements trs = table.select("tr");
            for (int i = 1; i < trs.size(); ++i) {
                Element tr = trs.get(i);
                Elements tds = tr.select("td");
                if (!tds.get(4).text().equals("Pending") && !tds.get(4).text().equals("Running") ) {
                    continue;
                }
                Date d = DateUtil.StringToDate(tds.get(2).text(), "yyyy-MM-dd HH:mm:ss");
                RecentContest contest = new RecentContest();
                contest.setName(tds.get(1).text());
                contest.setLink("http://acm.hdu.edu.cn" + tds.get(1).select("a[href]").attr("href"));
                contest.setOj("HDU");
                contest.setAccess(tds.get(3).text());
                contest.setStart_date(d);
                contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                contest.setWeek(DateUtil.getWeek(d));
                contest.setEnd_date(contest.getStart_date());
                contest.setEnd_time(contest.getStart_time());
                contestList.add(contest);
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectAtCoderContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Document document = Jsoup.connect("https://atcoder.jp/contests/").get();
            Elements div = document.select("#contest-table-upcoming");
            Elements table = div.select("table.table");
            Elements trs = table.select("tr");
            Elements divAction = document.select("#contest-table-action");
            if (divAction.size() > 0) {
                Elements tableAction = divAction.select("table.table");
                Elements trsAction = tableAction.select("tr");
                if (trsAction.size() > 1) {
                    trsAction.remove(0);
                    trs.addAll(trsAction);
                }
            }

            for (int i = 1; i < trs.size(); ++i) {
                Element tr = trs.get(i);
                Elements tds = tr.select("td");
                RecentContest contest = new RecentContest();
                Date d = DateUtil.StringToDate(tds.get(0).text(), "yyyy-MM-dd HH:mm:ss");
                Calendar calendar = Calendar.getInstance();
                calendar.setTime(d);
                calendar.add(calendar.HOUR, -1);
                SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd HH:mm");
                d = calendar.getTime();
                contest.setStart_time(fmt.format(d));
                contest.setStart_date(d);
                contest.setWeek(DateUtil.getWeek(d));

                String duration[] = tds.get(2).text().split(":");
                Date dateEnd = new Date(d.getTime() + Integer.parseInt(duration[0])*60*60*1000 + Integer.parseInt(duration[1])*60*1000);
                contest.setEnd_date(dateEnd);
                contest.setEnd_time(fmt.format(dateEnd));

                contest.setName(tds.get(1).text());
                contest.setLink("https://atcoder.jp" + tds.get(1).select("a[href]").attr("href"));
                contest.setOj("AtCoder");
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectCodeForcesContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Connection conn = Jsoup.connect("https://codeforces.com/contests").timeout(10000);
            conn.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            conn.header("Accept-Encoding", "gzip, deflate, sdch");
            conn.header("Accept-Language", "zh-CN,zh;q=0.8");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

            Document document = conn.get();
            Elements div = document.select("div.contestList");
            Elements divContests = div.select("div.datatable");
            Elements table = divContests.select("table");
            Elements trs = table.get(0).select("tr");
            for (int i = 1; i < trs.size(); ++i) {
                Element tr = trs.get(i);
                Elements tds = tr.select("td");
                RecentContest contest = new RecentContest();
                contest.setName(tds.get(0).text());
                SimpleDateFormat format = new SimpleDateFormat("MMM/dd/yyyy HH:mm", Locale.UK);
                Date date = format.parse(tds.get(2).text());
                Calendar calendar = Calendar.getInstance();
                calendar.setTime(date);
                calendar.add(calendar.HOUR, 5);
                date = calendar.getTime();
                contest.setStart_time(DateUtil.DateToString(date,"yyyy-MM-dd HH:mm"));

                String duration[] = tds.get(3).text().split(":");
                Date dateEnd = new Date(date.getTime() + Integer.parseInt(duration[0])*60*60*1000 + Integer.parseInt(duration[1])*60*1000);
                contest.setEnd_date(dateEnd);
                contest.setEnd_time(DateUtil.DateToString(dateEnd,"yyyy-MM-dd HH:mm"));

                contest.setLink("https://codeforces.com/contests");
                contest.setOj("CodeForces");
                contest.setAccess("");
                contest.setWeek(DateUtil.getWeek(date));
                contest.setStart_date(date);
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        } catch (ParseException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectHappyOJContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Connection conn = Jsoup.connect("https://happyoj.com/contests").timeout(10000);
            conn.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            conn.header("Accept-Encoding", "gzip, deflate, sdch");
            conn.header("Accept-Language", "zh-CN,zh;q=0.8");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

            Document document = conn.get();
            Elements table = document.select("table.contest_local_upcoming");
            Elements trs = table.get(0).select("tr");
            for (int i = 1; i < trs.size(); ++i) {
                Element tr = trs.get(i);
                Elements tds = tr.select("td");
                if (tds.size() < 3) {
                    continue;
                }
                RecentContest contest = new RecentContest();
                contest.setName(tds.get(0).text());
                Date d = DateUtil.StringToDate(tds.get(1).text(), "yyyy-MM-dd HH:mm");
                contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                contest.setStart_date(d);
                String duration[] = tds.get(2).text().split(":");
                Date dateEnd = new Date(d.getTime() + Integer.parseInt(duration[0])*60*60*1000 + Integer.parseInt(duration[1])*60*1000);
                contest.setEnd_date(dateEnd);
                contest.setEnd_time(DateUtil.DateToString(dateEnd,"yyyy-MM-dd HH:mm"));
                contest.setLink("https://happyoj.com/contests");
                contest.setOj("HappyOJ");
                contest.setAccess("");
                contest.setWeek(DateUtil.getWeek(d));
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectLeetCodeContests() {
        try {
            log.info("Collect leetcode contest.");
            List<RecentContest> contestList = new ArrayList<>();
            Document document = getPageWaitJS("https://leetcode.cn/contest/");
            log.info("Collect leetcode contest getPageWaitJS.");
            Elements contestsDiv = document.select("div.contest-cards-base");
            Elements contests = contestsDiv.select("div.contest-card-base");

            for (int i = 0; i < contests.size(); ++i) {
                Element c = contests.get(i);
                RecentContest contest = new RecentContest();
                contest.setOj("LeetCode");
                contest.setLink("https://leetcode.cn" + c.select("a[href]").attr("href"));

                Pattern p = Pattern.compile("(\\d{4})-(\\d{1,2})-(\\d{1,2})/(\\d{1,2}):(\\d{1,2})");
                String time = new String();
                if (c.select("div.contest-upcoming").size() > 0) {
                    contest.setName(c.select("div.card-title").text());
                    time = c.select("div.time").text()
                            .replaceAll(" +","")
                            .replaceAll("：", "/");
                } else if (c.select("div.biweekly-ongoing").size() > 0) {
                    contest.setName(c.select("div.title").text());
                    time = c.select("div.contest-time").text()
                            .replaceAll(" +", "")
                            .replaceAll("：", "/");
                } else if (c.select("div.contest-ongoing").size() > 0) {
                    contest.setName(c.select("div.card-title").text());
                    time = c.select("div.time").text()
                            .replaceAll(" +","")
                            .replaceAll("：", "/");
                } else {
                    continue;
                }

                /*
                * 中国时间/2024-04-28/10:30~12:00
                * 中国时间/2024-04-27/22:30~2024-04-28/00:00
                * */
                String timeContest[] = time.replaceAll("[\u4e00-\u9fa5]", "").substring(1).split("~");
                Date d = DateUtil.StringToDate(timeContest[0], "yyyy-MM-dd/HH:mm");
                contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                contest.setWeek(DateUtil.getWeek(d));
                contest.setStart_date(d);

                String dateEndStr = timeContest[1];
                if (dateEndStr.length() < 16) {
                    dateEndStr = timeContest[0].split("/")[0] + "/" + dateEndStr;
                }
                Date dateEnd = DateUtil.StringToDate(dateEndStr, "yyyy-MM-dd/HH:mm");
                contest.setEnd_time(DateUtil.DateToString(dateEnd,"yyyy-MM-dd HH:mm"));
                contest.setEnd_date(dateEnd);
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectBaekJoonContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();

            Connection conn = Jsoup.connect("https://www.acmicpc.net/contest/official/list").timeout(10000);
            conn.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            conn.header("Accept-Encoding", "gzip, deflate, sdch");
            conn.header("Accept-Language", "zh-CN,zh;q=0.8");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

            Document document = conn.get();
            Elements table = document.select("table.table");
            if (table.size() == 0) {
                return null;
            }
            Elements trs = table.get(0).select("tr.info");
            for (int i = 0; i < trs.size(); ++i) {
                Element tr = trs.get(i);
                Elements tds = tr.select("td");
                RecentContest contest = new RecentContest();
                contest.setName(tds.get(0).text());
                Pattern p = Pattern.compile("(\\d{4})(\\s)(\\d{1,2})(\\s)(\\d{1,2})(\\s)(\\d{1,2}):(\\d{1,2})");
                Matcher m = p.matcher(tds.get(3).text().replaceAll("[^0-9 :]", ""));
                if(!m.find()) {
                    continue;
                }
                Date d = DateUtil.StringToDate(m.group(0), "yyyy MM dd HH:mm");
                contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                contest.setStart_date(d);
                contest.setWeek(DateUtil.getWeek(d));
                m = p.matcher(tds.get(4).text().replaceAll("[^0-9 :]", ""));
                if(!m.find()) {
                    continue;
                }
                Date dateEnd = DateUtil.StringToDate(m.group(0), "yyyy MM dd HH:mm");
                contest.setEnd_time(DateUtil.DateToString(dateEnd,"yyyy-MM-dd HH:mm"));
                contest.setEnd_date(dateEnd);

                contest.setLink("https://www.acmicpc.net" + tds.get(0).select("a[href]").attr("href"));
                contest.setOj("BaekJoon");
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectVnoiContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Connection.Response response = Jsoup.connect("https://oj.vnoi.info/accounts/login/?next=/contests/")
                    .method(Connection.Method.GET)
                    .execute();
            log.info(response.cookies().toString());
            Connection conn = Jsoup.connect("https://oj.vnoi.info/contests/?hide_private_contests=false").timeout(10000)
                    .ignoreContentType(true)
                    .ignoreHttpErrors(true);
            Document document = conn.userAgent("Chrome").get();
            log.info(document.toString());
            Elements table = document.select("table.contest-list");
            log.info("contest_list:" + table.size());
            Elements tbody = table.get(0).select("tbody");
            log.info("contest_list tbody:" + tbody.size());
            Elements trs = table.get(0).select("tr.info");
            for (int i = 0; i < trs.size(); ++i) {
                Element tr = trs.get(i);
                Elements tds = tr.select("td");
                RecentContest contest = new RecentContest();
                contest.setName(tds.get(0).text());
                Pattern p = Pattern.compile("(\\d{4})(\\s)(\\d{1,2})(\\s)(\\d{1,2})(\\s)(\\d{1,2}):(\\d{1,2})");
                Matcher m = p.matcher(tds.get(3).text().replaceAll("[^0-9 :]", ""));
                if(!m.find()) {
                    continue;
                }
                Date d = DateUtil.StringToDate(m.group(0), "yyyy MM dd HH:mm");
                contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm:ss"));
                contest.setWeek(DateUtil.getWeek(d));
                contest.setStart_date(d);
                contest.setLink("https://www.acmicpc.net" + tds.get(0).select("a[href]").attr("href"));
                contest.setOj("Baekjoon");
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectDmojContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Connection conn = Jsoup.connect("https://dmoj.ca/contests").timeout(10000);
            conn.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            conn.header("Accept-Encoding", "gzip, deflate, sdch");
            conn.header("Accept-Language", "zh-CN,zh;q=0.8");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

            Document document = conn.get();
            Elements table = document.select("table.contest-list");
            for (int j = 0; j < table.size() - 1; ++j) {
                Elements tbody = table.get(j).select("tbody");
                Elements trs = tbody.get(0).select("tr");
                for (int i = 0; i < trs.size(); ++i) {
                    Element tr = trs.get(i);
                    Elements tds = tr.select("td");
                    Element td = tds.get(0);
                    Elements titleE = td.select("a.contest-list-title");
                    RecentContest contest = new RecentContest();
                    contest.setName(titleE.get(0).text());
                    Elements time = td.select("div.time");
                    String[] ds = time.text().split("-");
                    SimpleDateFormat formatter = new SimpleDateFormat("yyyy MM dd HH:mm", Locale.ENGLISH);

                    Date d = formatter.parse(ds[0].trim().replaceAll("[^0-9 :-]", " "));
                    d.setTime(d.getTime() + 12 * 60 * 60 * 1000);
                    contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setStart_date(d);

                    Date dateEnd = formatter.parse(ds[1].trim().replaceAll("[^0-9 :-]", " "));
                    dateEnd.setTime(dateEnd.getTime() + 12 * 60 * 60 * 1000);
                    contest.setEnd_time(DateUtil.DateToString(dateEnd,"yyyy-MM-dd HH:mm"));
                    contest.setEnd_date(dateEnd);

                    contest.setLink("https://dmoj.ca" + tds.get(0).select("a[href]").attr("href"));
                    contest.setOj("DMOJ");
                    contest.setAccess("");
                    contest.setWeek(DateUtil.getWeek(d));
                    contestList.add(contest);
                    //log.info(contest.toString());
                }
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        } catch (ParseException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectNowcoderContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();

            Connection conn = Jsoup.connect("https://ac.nowcoder.com/acm/contest/vip-index").timeout(10000);
            Document document = conn.get();
            Elements div = document.select("div.js-current");
            Elements contests = div.select("div.js-item");
            for (int i = 0; i < contests.size(); i++) {
                RecentContest contest = new RecentContest();
                Elements item = contests.get(i).select("div.platform-item-cont");
                Elements titles = item.get(0).select("a");
                contest.setName(titles.get(0).text());
                contest.setLink("https://ac.nowcoder.com" + titles.get(0).select("a[href]").attr("href"));
                Elements time = item.get(0).select("ul.platform-info li.match-time-icon");
                String[] ds = time.text().split("至");
                Date d = DateUtil.StringToDate(ds[0].replaceAll("[^0-9 :-]", ""), "yyyy-MM-dd HH:mm");
                contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                contest.setStart_date(d);
                Date dateEnd = DateUtil.StringToDate(ds[1].replaceAll("[^0-9 :-]", ""), "yyyy-MM-dd HH:mm");
                contest.setEnd_time(DateUtil.DateToString(dateEnd,"yyyy-MM-dd HH:mm"));
                contest.setEnd_date(dateEnd);
                contest.setOj("NowCoder");
                contest.setAccess("");
                contest.setWeek(DateUtil.getWeek(d));
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectAcWingContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Connection conn = Jsoup.connect("https://www.acwing.com/activity/1/competition/").timeout(10000);
            Document document = conn.get();
            Elements contests = document.select("div.activity-index-block");
            //log.info("num " + contests.size());
            for (int i = 0; i < contests.size(); i++) {
                RecentContest contest = new RecentContest();
                Elements item = contests.get(i).select("div.col-md-11");
                contest.setLink("https://www.acwing.com" + item.get(0).select("a[href]").attr("href"));
                Elements titles = item.get(0).select("span.activity_abstract");
                contest.setName(titles.get(0).text());
                Elements status = item.get(0).select("span.activity_status");
                if (status.get(0).text().contains("已结束")) {
                    break;
                }
                Elements center = item.get(0).select("div.text-center");
                Elements timeDiv = center.get(0).select("div.col-xs-6");
                Elements time = timeDiv.get(1).select("span.activity_td");
                Date d = DateUtil.StringToDate(time.get(0).text(), "yyyy-MM-dd HH:mm:ss");
                contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                contest.setStart_date(d);
                contest.setWeek(DateUtil.getWeek(d));
                String html = Jsoup.connect(contest.getLink()).timeout(10000).get().toString();
                int index = html.indexOf("let end_time = Date.parse(\"");
                if (index > -1) {
                    index = index + "let end_time = Date.parse(\"".length();
                    String dateEndStr = html.substring(index, index + 19) ;
                    Date dateEnd = DateUtil.StringToDate(dateEndStr, "yyyy-MM-dd HH:mm:ss");
                    contest.setEnd_time(DateUtil.DateToString(dateEnd,"yyyy-MM-dd HH:mm"));
                    contest.setEnd_date(dateEnd);
                } else {
                    contest.setEnd_time(contest.getStart_time());
                    contest.setEnd_date(contest.getStart_date());
                }

                contest.setOj("AcWing");
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectEolympContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Connection conn = Jsoup.connect("https://www.eolymp.com/en/contests").timeout(10000);
            Document document = conn.get();

            Elements contests = document.select("div.eo-list__item");
            for (int i = 0; i < contests.size(); i++) {
                RecentContest contest = new RecentContest();
                if (contests.get(i).select("div.eo-competition-row__status_pending").size() == 0
                    && contests.get(i).select("div.eo-competition-row__status_inprogress").size() == 0) {
                    break;
                }

                contest.setLink(contests.get(i).select("a[href]").attr("href"));
                Elements titles = contests.get(i).select("a.eo-competition-row__name");
                contest.setName(titles.get(0).text());
                Elements titles_tip = contests.get(i).select("span.eo-competition-row__visibility");
                if (titles_tip.size() > 0) {
                    contest.setName(contest.getName().replaceAll(titles_tip.text(), ""));
                }

                Elements dates = contests.get(i).select("div.eo-competition-row__dates");
                Elements dateElmts = dates.get(0).select("div");
                if (dateElmts.size() > 1) {
                    String dateStartStr = dateElmts.get(1).text();
                    String dateEndStr = dateElmts.get(1).text();
                    if (dateStartStr.length() < 11) {
                        String timeString = dateElmts.get(2).text();
                        String[] ts = timeString.split("-");
                        dateStartStr = dateStartStr + " " + ts[0];
                        dateEndStr = dateStartStr + " " + ts[1];
                    } else {
                        dateEndStr = dateElmts.get(2).text();;
                    }

                    Date d = DateUtil.StringToDate(dateStartStr, "dd.MM.yyy HH:mm");
                    d.setTime(d.getTime() + 6 * 60 * 60 * 1000);
                    contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setStart_date(d);
                    contest.setWeek(DateUtil.getWeek(d));

                    Date dateEnd = DateUtil.StringToDate(dateEndStr, "dd.MM.yyy HH:mm");
                    dateEnd.setTime(dateEnd.getTime() + 6 * 60 * 60 * 1000);
                    contest.setStart_time(DateUtil.DateToString(dateEnd,"yyyy-MM-dd HH:mm"));
                    contest.setStart_date(dateEnd);
                }

                contest.setOj("Eolymp");
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectContestsFromClist(String ojName, String url) {
        List<RecentContest> contestList = new ArrayList<>();
        try {
            //log.info(url);
            Connection conn = Jsoup.connect(url).timeout(15000);
            Document document = conn.get();
            Elements contestsDiv = document.select("#contests");
            if (contestsDiv.size() == 0) {
                return contestList;
            }
            //log.info(contestsDiv.toString());
            Elements contests = contestsDiv.get(0).select("div.contest");
            //log.info("contests num: " + contests.size());
            for (int i = 0; i < contests.size(); i++) {
                RecentContest contest = new RecentContest();
                String dataJson = contests.get(i).select("a.data-ace").attr("data-ace");
                JSONObject jsonObject = JSONObject.parseObject(new String(dataJson));
                contest.setName(jsonObject.get("title").toString());
                JSONObject jsonDateObject = JSONObject.parseObject(new String(jsonObject.get("time").toString()));
                Elements titles = contests.get(i).select("a.title_search");
                contest.setLink(titles.get(0).select("a[href]").attr("href"));
                try {
                    SimpleDateFormat dateFormat = new SimpleDateFormat("MMMM dd, yyyy HH:mm:ss", Locale.ENGLISH);
                    Date d = dateFormat.parse(jsonDateObject.get("start").toString());
                    /* UTC+8 */
                    d.setTime(d.getTime() + 8 * 60 * 60 * 1000);
                    contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setWeek(DateUtil.getWeek(d));
                    contest.setStart_date(d);

                    Date dateEnd = dateFormat.parse(jsonDateObject.get("end").toString());
                    /* UTC+8 */
                    dateEnd.setTime(dateEnd.getTime() + 8 * 60 * 60 * 1000);
                    contest.setEnd_time(DateUtil.DateToString(dateEnd,"yyyy-MM-dd HH:mm"));
                    contest.setEnd_date(dateEnd);
                } catch (ParseException e) {
                    continue;
                }
                contest.setOj(ojName);
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return contestList;
        }
    }
    public static List<RecentContest> CollectTopCoderContests() {
        List<RecentContest> contestList = new ArrayList<>();
        try {
            Connection conn = Jsoup.connect("https://www.topcoder.com/community/events")
                .timeout(10000).
                ignoreContentType(true);
            Document document = conn.get();
            Pattern pattern = Pattern.compile("<a[^>]*href=\"(https://calendar.google.com/[^\"]*)\"[^>]*>"); // 正则表达式
            Matcher matcher = pattern.matcher(document.toString());
            while (matcher.find()) {
                RecentContest contest = new RecentContest();
                String str = java.net.URLDecoder.decode(matcher.group(1).toString(), "utf-8");;
                String[] info = str.split(";");
                /* dates=20240425T150000Z/20240425T170000Z */
                String timeStart = info[1].substring(6, 21).replaceAll("T", " ");
                try {
                    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd HHmmss");
                    Date d = sdf.parse(timeStart);
                    d.setTime(d.getTime() + 8 * 60 * 60 * 1000);
                    contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setWeek(DateUtil.getWeek(d));
                    contest.setStart_date(d);
                } catch (ParseException e) {
                    log.error(e.getMessage());
                    continue;
                }
                String timeEnd = info[1].substring(23, 38).replaceAll("T", " ");
                try {
                    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd HHmmss");
                    Date d = sdf.parse(timeEnd);
                    d.setTime(d.getTime() + 8 * 60 * 60 * 1000);
                    contest.setEnd_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setEnd_date(d);
                } catch (ParseException e) {
                    log.error(e.getMessage());
                    continue;
                }
                if (contest.getEnd_date().getTime() < new Date().getTime()) {
                    continue;
                }
                contest.setLink("https://www.topcoder.com/community/events");
                contest.setName(info[4].substring(5));
                contest.setOj("TopCoder");
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
        return contestList;
    }
    public static List<RecentContest> CollectCodeChefContests() {
        List<RecentContest> contestList = new ArrayList<>();
        String url = "https://www.codechef.com/api/list/contests/all";
        Map<String,String> headers = new HashMap<>(1,1);
        headers.put("accept", "application/json");
        String jsonS = HttpClientUtil.doGet(url, headers);
        JSONObject json = JSONObject.parseObject(new String(jsonS));
        JSONArray listPresent = JSONObject.parseArray(new String(json.get("present_contests").toString()));
        JSONArray list = JSONObject.parseArray(new String(json.get("future_contests").toString()));
        list.addAll(listPresent);
        for (int i = 0; i < list.size(); i++) {
            RecentContest contest = new RecentContest();
            JSONObject contestE = list.getJSONObject(i);
            String timeStart = contestE.get("contest_start_date_iso").toString()
                    .substring(0, 19).replaceAll("T", " ");
            try {
                SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                Date d = sdf.parse(timeStart);
                /* india to  */
                d.setTime(d.getTime() + 2 * 60 * 60 * 1000 + 30 * 60 * 1000);
                contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                contest.setWeek(DateUtil.getWeek(d));
                contest.setStart_date(d);
            } catch (ParseException e) {
                log.error(e.getMessage());
                continue;
            }
            String timeEnd = contestE.get("contest_end_date_iso").toString()
                    .substring(0, 19).replaceAll("T", " ");
            try {
                SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                Date d = sdf.parse(timeEnd);
                /* india to  */
                d.setTime(d.getTime() + 2 * 60 * 60 * 1000 + 30 * 60 * 1000);
                contest.setEnd_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                contest.setEnd_date(d);
            } catch (ParseException e) {
                log.error(e.getMessage());
                continue;
            }
            contest.setLink("https://www.codechef.com/"+contestE.get("contest_code").toString());
            contest.setName(contestE.get("contest_name").toString());
            contest.setOj("CodeChef");
            contest.setAccess("");
            contestList.add(contest);
            //log.info(contest.toString());
        }
        return contestList;
    }
    public static List<RecentContest> CollectLuoGuContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Connection conn = Jsoup.connect("https://www.luogu.com.cn/contest/list?page=1&_contentOnly=1")
                    .timeout(10000).
                    ignoreContentType(true);
            conn.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            conn.header("Accept-Encoding", "gzip, deflate, sdch");
            conn.header("Accept-Language", "zh-CN,zh;q=0.8");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

            Document document = conn.get();
            Elements jsonE = document.select("body");
            JSONObject jsonObject = JSONObject.parseObject(new String(jsonE.text()));
            JSONObject jsonObjectContestE = JSONObject.parseObject(new String(jsonObject.get("currentData").toString()));
            JSONObject resultE = JSONObject.parseObject(new String(jsonObjectContestE.get("contests").toString()));
            JSONArray list = resultE.getJSONArray("result");
            for (int i = 0; i < list.size(); i++) {
                RecentContest contest = new RecentContest();
                JSONObject contestE = list.getJSONObject(i);
                Date dt = new Date();
                long dtEnd = Long.parseLong(contestE.get("endTime").toString());
                if ((long)(dt.getTime()/1000) > dtEnd) {
                    break;
                }
                long dtStart = Long.parseLong(contestE.get("startTime").toString());
                SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                String dtStr = sdf.format(new Date(dtStart * 1000));
                try {
                    Date d = sdf.parse(dtStr);
                    contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setWeek(DateUtil.getWeek(d));
                    contest.setStart_date(d);
                } catch (ParseException e) {
                    log.error(e.getMessage());
                    continue;
                }
                String dtEndStr = sdf.format(new Date(dtEnd * 1000));
                try {
                    Date d = sdf.parse(dtEndStr);
                    contest.setEnd_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setEnd_date(d);
                } catch (ParseException e) {
                    log.error(e.getMessage());
                    continue;
                }
                contest.setLink("https://www.luogu.com.cn/contest/"+contestE.get("id").toString());
                contest.setName(contestE.get("name").toString());
                contest.setOj("洛谷");
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectLightOJContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Connection conn = Jsoup.connect("https://lightoj.com/api/v1/contests?page=1&contestType=currentOrUpcoming")
                    .timeout(10000).
                    ignoreContentType(true);
            conn.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            conn.header("Accept-Encoding", "gzip, deflate, sdch");
            conn.header("Accept-Language", "zh-CN,zh;q=0.8");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

            Document document = conn.get();
            Elements jsonE = document.select("body");
            JSONObject jsonObject = JSONObject.parseObject(new String(jsonE.text()));
            JSONObject jsonObjectContestE = JSONObject.parseObject(new String(jsonObject.get("data").toString()));
            JSONObject resultE = JSONObject.parseObject(new String(jsonObjectContestE.get("contests").toString()));
            JSONArray list = resultE.getJSONArray("data");
            for (int i = 0; i < list.size(); i++) {
                RecentContest contest = new RecentContest();
                JSONObject contestE = list.getJSONObject(i);
                /* 2024-04-21T09:00:00.000Z */
                String timeStart = contestE.get("contestStartTimestamp").toString()
                        .substring(0, 19).replaceAll("T", " ");
                try {
                    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                    Date d = sdf.parse(timeStart);
                    /* UTC+8 */
                    d.setTime(d.getTime() + 8 * 60 * 60 * 1000);
                    contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setWeek(DateUtil.getWeek(d));
                    contest.setStart_date(d);
                } catch (ParseException e) {
                    log.error(e.getMessage());
                    continue;
                }
                String timeEnd = contestE.get("contestEndTimestamp").toString()
                        .substring(0, 19).replaceAll("T", " ");
                try {
                    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                    Date d = sdf.parse(timeEnd);
                    d.setTime(d.getTime() + 8 * 60 * 60 * 1000);
                    contest.setEnd_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setEnd_date(d);
                } catch (ParseException e) {
                    log.error(e.getMessage());
                    continue;
                }
                contest.setLink("https://lightoj.com/contest/"+contestE.get("contestHandleStr").toString());
                contest.setName(contestE.get("contestTitleStr").toString());
                contest.setOj("LightOJ");
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectProjectEulerContests() {
        List<RecentContest> contestList = new ArrayList<>();
        List<RecentContest> contestOngoing = CollectContestsFromClist("ProjectEuler","https://clist.by/?resource=65&view=list&group=no&status=running");
        List<RecentContest> contestComing = CollectContestsFromClist("ProjectEuler","https://clist.by/?resource=65&view=list&group=no&status=coming");
        if (contestComing.size() > 0) {
            contestList.addAll(contestComing);
        }
        if (contestComing.size() > 0) {
            contestList.addAll(contestOngoing);
        }
        return contestList;
    }
    public static List<RecentContest> CollectLanQiaoContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Connection conn = Jsoup.connect("https://www.lanqiao.cn/api/v2/contests/?sort=opentime&paginate=0&status=not_finished&game_type_code=2")
                    .timeout(10000).
                    ignoreContentType(true);
            conn.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            conn.header("Accept-Encoding", "gzip, deflate, sdch");
            conn.header("Accept-Language", "zh-CN,zh;q=0.8");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

            Document document = conn.get();
            Elements jsonE = document.select("body");
            JSONArray list = JSONObject.parseArray(new String(jsonE.text()));
            for (int i = 0; i < list.size(); i++) {
                RecentContest contest = new RecentContest();
                JSONObject contestE = list.getJSONObject(i);
                /* 2024-04-21T09:00:00.000Z */
                String timeStart = contestE.get("open_at").toString()
                        .substring(0, 19).replaceAll("T", " ");
                try {
                    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                    Date d = sdf.parse(timeStart);
                    /* UTC+8 */
                    //d.setTime(d.getTime() + 8 * 60 * 60 * 1000);
                    contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setWeek(DateUtil.getWeek(d));
                    contest.setStart_date(d);
                } catch (ParseException e) {
                    log.error(e.getMessage());
                    continue;
                }
                String timeEnd = contestE.get("end_at").toString()
                        .substring(0, 19).replaceAll("T", " ");
                try {
                    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                    Date d = sdf.parse(timeEnd);
                    contest.setEnd_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setEnd_date(d);
                } catch (ParseException e) {
                    log.error(e.getMessage());
                    continue;
                }
                contest.setLink("https://www.lanqiao.cn"+contestE.get("html_url").toString());
                contest.setName(contestE.get("name").toString());
                contest.setOj("蓝桥");
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectJSKContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Connection conn = Jsoup.connect("https://www.jisuanke.com/api/contests?page=1")
                    .timeout(10000).
                    ignoreContentType(true);
            conn.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            conn.header("Accept-Encoding", "gzip, deflate, sdch");
            conn.header("Accept-Language", "zh-CN,zh;q=0.8");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

            Document document = conn.get();
            Elements jsonE = document.select("body");
            JSONObject jsonObject = JSONObject.parseObject(new String(jsonE.text()));
            JSONObject jsonObjectContestPresent = JSONObject.parseObject(new String(jsonObject.get("present").toString()));
            JSONArray listPresent = JSONObject.parseArray(new String(jsonObjectContestPresent.get("contests").toString()));
            JSONObject jsonObjectContestUpcoming = JSONObject.parseObject(new String(jsonObject.get("upcoming").toString()));
            JSONArray list = JSONObject.parseArray(new String(jsonObjectContestUpcoming.get("contests").toString()));
            list.addAll(listPresent);
            for (int i = 0; i < list.size(); i++) {
                RecentContest contest = new RecentContest();
                JSONObject contestE = list.getJSONObject(i);
                String timeStart = contestE.get("startTime").toString();
                try {
                    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                    Date d = sdf.parse(timeStart);
                    contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setWeek(DateUtil.getWeek(d));
                    contest.setStart_date(d);
                    String duration = contestE.get("duration").toString();
                    Date dateEnd = new Date(d.getTime() + Integer.parseInt(duration)*60*1000);
                    contest.setEnd_date(dateEnd);
                    contest.setEnd_time(DateUtil.DateToString(dateEnd,"yyyy-MM-dd HH:mm"));
                } catch (ParseException e) {
                    log.error(e.getMessage());
                    continue;
                }
                contest.setLink("https://www.jisuanke.com/contest/"+contestE.get("contestId").toString());
                contest.setName(contestE.get("title").toString());
                contest.setOj("计蒜客");
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectTophContestsEx() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            String url = "https://toph.co/contests/current";
            Map<String,String> headers = new HashMap<>(4,4);
            headers.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7");
            headers.put("Accept-Encoding", "gzip, deflate, br, zstd");
            headers.put("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
            headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36");
            String jsonS = HttpClientUtil.doGet(url, headers);
            log.info("####"+jsonS);
            //Document document = getPageWaitJS("https://toph.co/contests/current");
            Connection conn = Jsoup.connect("https://toph.co/contests/current").timeout(5000);
            conn.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7");
            conn.header("Accept-Encoding", "gzip, deflate, br, zstd");
            conn.header("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36");

            Document document = conn.get();
            Elements table = document.select("table.table");
            Elements contests = table.select("tr");
            log.info("size="+contests);
            for (int i = 0; i < contests.size(); i++) {
                RecentContest contest = new RecentContest();
                Elements item = contests.get(i).select("div");
                Elements titles = item.get(0).select("a");
                log.info(titles.get(0).text());
                contest.setName(titles.get(0).text());
                contest.setLink("https://toph.co" + titles.get(0).select("a[href]").attr("href"));
                String time = item.get(1).select("span.timestamp").attr("data-timestamp");
                log.info(time);
                Date d = new Date(Integer.parseInt(time));
                contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                contest.setWeek(DateUtil.getWeek(d));
                contest.setStart_date(d);
                contest.setOj("Toph");
                contest.setAccess("");
                contestList.add(contest);
                log.info(contest.toString());
            }
            return contestList;
        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectTophContests() {
        List<RecentContest> contestList = new ArrayList<>();
        List<RecentContest> contestComing = CollectContestsFromClist("Toph","https://clist.by/?resource=110&view=list&group=no&status=coming");
        List<RecentContest> contestOngoing = CollectContestsFromClist("Toph","https://clist.by/?resource=110&view=list&group=no&status=running");
        if (contestComing.size() > 0) {
            contestList.addAll(contestComing);
        }
        if (contestOngoing.size() > 0) {
            contestList.addAll(contestOngoing);
        }
        return contestList;
    }
    public static List<RecentContest> CollectHuaweiContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Connection conn = Jsoup.connect("https://competition.huaweicloud.com/v5/noauth/competitions/list?category_id=10001&track_id=&offset=0&limit=10&status=active&_=1713668421852")
                    .timeout(10000).ignoreContentType(true);
            conn.header("Accept", "application/json, text/plain, */*");
            conn.header("Accept-Encoding", "gzip, deflate, br, zstd");
            conn.header("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
            conn.header("Cache-Control", "no-cache");
            conn.header("Connection", "Connection");
            conn.header("Content-Type", "application/json; charset=utf-8");
            conn.header("Host", "competition.huaweicloud.com");
            conn.header("Pragma", "no-cache");
            conn.header("Referer", "https://competition.huaweicloud.com/mobile/competitions");
            conn.header("Sec-Fetch-Dest", "empty");
            conn.header("Sec-Fetch-Mode", "cors");
            conn.header("Sec-Fetch-Site", "no-cache");
            conn.header("Pragma", "same-origin");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");
            conn.header("X-Language", "zh-cn");
            conn.header("language", "zh-cn3");
            conn.header("projectname", "");
            conn.header("sec-ch-ua", "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"");
            conn.header("sec-ch-ua-mobile", "?0");
            conn.header("sec-ch-ua-platform", "\"Windows\"");
            conn.header("x-requested-with", "XMLHttpRequest");

            Document document = conn.get();
            Elements jsonE = document.select("body");
            JSONObject jsonObject = JSONObject.parseObject(new String(jsonE.text()));
            JSONArray list = JSONObject.parseArray(new String(jsonObject.get("results").toString()));
            for (int i = 0; i < list.size(); i++) {
                RecentContest contest = new RecentContest();
                JSONObject contestE = list.getJSONObject(i);
                String timeStart = contestE.get("start_time").toString()
                    .substring(0, 19).replaceAll("T", " ");
                try {
                    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                    Date d = sdf.parse(timeStart);
                    contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setWeek(DateUtil.getWeek(d));
                    contest.setStart_date(d);
                } catch (ParseException e) {
                    log.error(e.getMessage());
                    continue;
                }
                String timeEnd = contestE.get("end_time").toString()
                        .substring(0, 19).replaceAll("T", " ");
                try {
                    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                    Date d = sdf.parse(timeEnd);
                    contest.setEnd_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setEnd_date(d);
                } catch (ParseException e) {
                    log.error(e.getMessage());
                    continue;
                }
                if (contestE.get("link_url").toString().length() > 0) {
                    contest.setLink(contestE.get("link_url").toString());
                } else {
                    contest.setLink("https://competition.huaweicloud.com/competitions");
                }
                contest.setName(contestE.get("title").toString());
                contest.setOj("Huawei");
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectNaukriContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            for (int page = 1; page <= 2; page++) {
                Connection conn = Jsoup.connect("https://api.codingninjas.com/api/v3/public_section/contest_list?page="+page)
                        .timeout(10000).ignoreContentType(true);
                conn.header("Accept", "application/json");
                conn.header("Accept-Encoding", "gzip, deflate, br, zstd");
                conn.header("Connection", "Keep-Alive");
                conn.header("Content-Type", "application/json; charset=utf-8");
                conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

                Document document = conn.get();
                Elements jsonE = document.select("body");
                JSONObject jsonObject = JSONObject.parseObject(new String(jsonE.text()));
                JSONObject dataObject = JSONObject.parseObject(new String(jsonObject.get("data").toString()));
                JSONArray list = JSONObject.parseArray(new String(dataObject.get("events").toString()));
                for (int i = 0; i < list.size(); i++) {
                    RecentContest contest = new RecentContest();
                    JSONObject contestE = list.getJSONObject(i);
                    Date dt = new Date();
                    long dtEnd = Long.parseLong(contestE.get("event_end_time").toString());
                    if ((long)(dt.getTime()/1000) > dtEnd) {
                        break;
                    }

                    String timeStart = contestE.get("event_start_time").toString();
                    Date d = new Date(Long.parseLong(timeStart)*1000);
                    contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                    contest.setWeek(DateUtil.getWeek(d));
                    contest.setStart_date(d);
                    String timeEnd = contestE.get("event_end_time").toString();
                    Date dateEnd = new Date(Long.parseLong(timeEnd)*1000);
                    contest.setEnd_time(DateUtil.DateToString(dateEnd,"yyyy-MM-dd HH:mm"));
                    contest.setEnd_date(dateEnd);
                    contest.setLink("https://www.naukri.com/code360/contests/"+contestE.get("slug").toString());
                    contest.setName(contestE.get("name").toString());
                    contest.setOj("Naukri");
                    contest.setAccess("");
                    contestList.add(contest);
                    //log.info(contest.toString());
                }
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectGeeksforgeeksContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            for (int page = 1; page <= 2; page++) {
                Connection conn = Jsoup.connect("https://practiceapi.geeksforgeeks.org/api/v1/events/?type=contest&page_number="+page+"&sub_type=all")
                        .timeout(10000).ignoreContentType(true);
                conn.header("Accept", "application/json");
                conn.header("Accept-Encoding", "gzip, deflate, br, zstd");
                conn.header("Connection", "Keep-Alive");
                conn.header("Content-Type", "application/json; charset=utf-8");
                conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

                Document document = conn.get();
                Elements jsonE = document.select("body");
                JSONObject jsonObject = JSONObject.parseObject(new String(jsonE.text()));
                JSONObject dataObject = JSONObject.parseObject(new String(jsonObject.get("results").toString()));
                JSONArray list = JSONObject.parseArray(new String(dataObject.get("upcoming").toString()));
                for (int i = 0; i < list.size(); i++) {
                    RecentContest contest = new RecentContest();
                    JSONObject contestE = list.getJSONObject(i);
                    String endStart = contestE.get("end_time").toString().replaceAll("T", " ");
                    try {
                        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                        Date d = sdf.parse(endStart);
                        d.setTime(d.getTime() + 2 * 60 * 60 * 1000 + 30 * 60 * 1000);
                        contest.setEnd_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                        contest.setEnd_date(d);
                    } catch (ParseException e) {
                        log.error(e.getMessage());
                        continue;
                    }
                    String timeStart = contestE.get("start_time").toString().replaceAll("T", " ");
                    try {
                        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                        Date d = sdf.parse(timeStart);
                        d.setTime(d.getTime() + 2 * 60 * 60 * 1000 + 30 * 60 * 1000);
                        contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                        contest.setWeek(DateUtil.getWeek(d));
                        contest.setStart_date(d);
                    } catch (ParseException e) {
                        log.error(e.getMessage());
                        continue;
                    }

                    contest.setLink("https://practice.geeksforgeeks.org/contest/"+contestE.get("slug").toString());
                    contest.setName(contestE.get("name").toString());
                    contest.setOj("Geeksforgeeks");
                    contest.setAccess("");
                    contestList.add(contest);
                    //log.info(contest.toString());
                }
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectKepContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            for (int page = 1; page <= 1; page++) {
                Connection conn = Jsoup.connect("https://kep.uz/api/contests?page="+page+"&page_size=10")
                        .timeout(10000).ignoreContentType(true);
                conn.header("Accept", "application/json");
                conn.header("Accept-Encoding", "gzip, deflate, br, zstd");
                conn.header("Connection", "Keep-Alive");
                conn.header("Content-Type", "application/json; charset=utf-8");
                conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

                Document document = conn.get();
                Elements jsonE = document.select("body");
                JSONObject jsonObject = JSONObject.parseObject(new String(jsonE.text()));
                JSONArray list = JSONObject.parseArray(new String(jsonObject.get("data").toString()));
                for (int i = 0; i < list.size(); i++) {
                    RecentContest contest = new RecentContest();
                    JSONObject contestE = list.getJSONObject(i);
                    int status = Integer.parseInt(contestE.get("status").toString());
                    if (status > 0) {
                        continue;
                    }
                    String endStart = contestE.get("finishTime").toString().substring(0,19).replaceAll("T", " ");
                    try {
                        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                        Date d = sdf.parse(endStart);
                        d.setTime(d.getTime() + 8 * 60 * 60 * 1000);
                        contest.setEnd_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                        contest.setEnd_date(d);
                    } catch (ParseException e) {
                        log.error(e.getMessage());
                        continue;
                    }
                    String timeStart = contestE.get("startTime").toString().substring(0,19).replaceAll("T", " ");
                    try {
                        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                        Date d = sdf.parse(timeStart);
                        d.setTime(d.getTime() + 8 * 60 * 60 * 1000);
                        contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                        contest.setWeek(DateUtil.getWeek(d));
                        contest.setStart_date(d);
                    } catch (ParseException e) {
                        log.error(e.getMessage());
                        continue;
                    }

                    contest.setLink("https://kep.uz/competitions/contests/contest/"+contestE.get("id").toString());
                    contest.setName(contestE.get("title").toString());
                    contest.setOj("KEP");
                    contest.setAccess("");
                    contestList.add(contest);
                    //log.info(contest.toString());
                }
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectCodeAnyContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Connection conn = Jsoup.connect("https://codeany.org/en/competitions").timeout(10000);
            conn.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            conn.header("Accept-Encoding", "gzip, deflate, sdch");
            conn.header("Accept-Language", "zh-CN,zh;q=0.8");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");
            Document document = conn.get();
            Elements contestsTbl = document.select("div.competition-table-body");
            Elements divs = contestsTbl.get(0).select("div.competition-table-row");
            for (int i = 0; i < divs.size(); ++i) {
                Element contestE = divs.get(i);
                Elements nameE = contestE.select("div.competition-name");
                RecentContest contest = new RecentContest();
                contest.setName(nameE.select("p").get(0).text());
                Elements dateE = contestE.select("div.competition-date");
                String startTime = dateE.select("p.time").get(0).text() + " " + dateE.select("p.hour").get(0).text();
                SimpleDateFormat dateFormat = new SimpleDateFormat("dd MMMM yyyy HH:mm", Locale.ENGLISH);
                Date d = dateFormat.parse(startTime);
                contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                contest.setWeek(DateUtil.getWeek(d));
                contest.setStart_date(d);
                Elements durationE = contestE.select("div.competition-duration");
                String REGEX = "[^(0-9)]";
                String duration = Pattern.compile(REGEX).matcher(durationE.text()).replaceAll("").trim();
                Date dateEnd = new Date(d.getTime() + Integer.parseInt(duration) * 60 * 1000);
                contest.setEnd_time(DateUtil.DateToString(dateEnd,"yyyy-MM-dd HH:mm"));
                contest.setEnd_date(dateEnd);
                if (dateEnd.getTime() < new Date().getTime()) {
                    break;
                }
                contest.setLink("https://codeany.org"+nameE.get(0).select("a[href]").attr("href"));
                contest.setOj("CodeAny");
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        } catch (ParseException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static List<RecentContest> CollectBsuirContests() {
        try {
            List<RecentContest> contestList = new ArrayList<>();
            Connection conn = Jsoup.connect("https://acm.bsuir.by/solve/api/v0/contests")
                    .timeout(10000).ignoreContentType(true);
            conn.header("Accept", "application/json");
            conn.header("Accept-Encoding", "gzip, deflate, br, zstd");
            conn.header("Connection", "Keep-Alive");
            conn.header("Content-Type", "application/json; charset=utf-8");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

            Document document = conn.get();
            Elements jsonE = document.select("body");
            JSONObject jsonObject = JSONObject.parseObject(new String(jsonE.text()));
            JSONArray list = JSONObject.parseArray(new String(jsonObject.get("contests").toString()));
            for (int i = 0; i < list.size(); i++) {
                RecentContest contest = new RecentContest();
                JSONObject contestE = list.getJSONObject(i);
                String timeStart = contestE.get("begin_time").toString();
                Date d = new Date(Long.parseLong(timeStart)*1000);
                contest.setStart_time(DateUtil.DateToString(d,"yyyy-MM-dd HH:mm"));
                contest.setWeek(DateUtil.getWeek(d));
                contest.setStart_date(d);
                Date dateEnd = new Date(d.getTime() + Integer.parseInt(contestE.get("duration").toString())*60*1000);
                contest.setEnd_time(DateUtil.DateToString(dateEnd, "yyyy-MM-dd HH:mm"));
                contest.setEnd_date(dateEnd);
                if (dateEnd.getTime() < new Date().getTime()) {
                    continue;
                }
                contest.setLink("https://acm.bsuir.by/solve/contests" + contestE.get("id").toString());
                contest.setName(contestE.get("title").toString());
                contest.setOj("Bsuir");
                contest.setAccess("");
                contestList.add(contest);
                //log.info(contest.toString());
            }
            return contestList;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
    public static void CollectContests() {
        Map map = new HashMap();
        List<OtherOJ>support_list = new ArrayList<>();
        support_list.add(new OtherOJ("HDU", "http://acm.hdu.edu.cn"));
        support_list.add(new OtherOJ("LeetCode", "https://leetcode.cn"));
        support_list.add(new OtherOJ("AtCoder", "https://atcoder.jp"));
        support_list.add(new OtherOJ("CodeForces", "https://codeforces.com"));
        support_list.add(new OtherOJ("HappyOJ", "https://happyoj.com"));
        support_list.add(new OtherOJ("BaekJoon", "https://www.acmicpc.net"));
        support_list.add(new OtherOJ("DMOJ", "https://dmoj.ca"));
        support_list.add(new OtherOJ("NowCoder", "https://ac.nowcoder.com"));
        support_list.add(new OtherOJ("AcWing", "https://www.acwing.com"));
        support_list.add(new OtherOJ("Eolymp", "https://www.eolymp.com"));
        support_list.add(new OtherOJ("TopCoder", "https://www.topcoder.com"));
        support_list.add(new OtherOJ("CodeChef", "https://www.codechef.com"));
        support_list.add(new OtherOJ("洛谷", "https://www.luogu.com.cn"));
        support_list.add(new OtherOJ("LightOJ", "https://lightoj.com"));
        support_list.add(new OtherOJ("ProjectEuler", "https://projecteuler.net"));
        support_list.add(new OtherOJ("蓝桥", "https://www.lanqiao.cn"));
        support_list.add(new OtherOJ("计蒜客", "https://www.jisuanke.com"));
        support_list.add(new OtherOJ("Toph", "https://toph.co"));
        support_list.add(new OtherOJ("Huawei", "https://competition.huaweicloud.com"));
        support_list.add(new OtherOJ("Naukri", "https://www.naukri.com/code360"));
        support_list.add(new OtherOJ("Geeksforgeeks", "https://geeksforgeeks.org"));
        support_list.add(new OtherOJ("KEP", "https://kep.uz"));
        support_list.add(new OtherOJ("CodeAny", "https://codeany.org"));
        support_list.add(new OtherOJ("Bsuir", "https://acm.bsuir.by"));

        List<RecentContest> contests = new ArrayList<>();
        List<RecentContest> contestsHdu = CollectHduContests();
        List<RecentContest> contestAtCoder = CollectAtCoderContests();
        List<RecentContest> contestCodeforces = CollectCodeForcesContests();
        List<RecentContest> contestHappyOJ = CollectHappyOJContests();
        List<RecentContest> contestBaekjoon = CollectBaekJoonContests();
        List<RecentContest> contestLeetCode = CollectLeetCodeContests();
        List<RecentContest> contestDmoj = CollectDmojContests();
        List<RecentContest> contestNowCoder = CollectNowcoderContests();
        List<RecentContest> contestAcWing = CollectAcWingContests();
        List<RecentContest> contestEolymp = CollectEolympContests();
        List<RecentContest> contestTopcoder = CollectTopCoderContests();
        List<RecentContest> contestCodeChef = CollectCodeChefContests();
        List<RecentContest> contestLuoGu = CollectLuoGuContests();
        List<RecentContest> contestLightOJ = CollectLightOJContests();
        List<RecentContest> contestPE = CollectProjectEulerContests();
        List<RecentContest> contestLanQiao = CollectLanQiaoContests();
        List<RecentContest> contestJiSuanKe = CollectJSKContests();
        List<RecentContest> contestToph = CollectTophContests();
        List<RecentContest> contestHuawei = CollectHuaweiContests();
        List<RecentContest> contestNaukri = CollectNaukriContests();
        List<RecentContest> contestGreeksforgeeks = CollectGeeksforgeeksContests();
        List<RecentContest> contestKep = CollectKepContests();
        List<RecentContest> contestCodeAny = CollectCodeAnyContests();
        List<RecentContest> contestbsuir = CollectBsuirContests();
        if (contestsHdu != null) {
            contests.addAll(contestsHdu);
        }
        if (contestAtCoder != null) {
            contests.addAll(contestAtCoder);
        }
        if (contestCodeforces != null) {
            contests.addAll(contestCodeforces);
        }
        if (contestHappyOJ != null) {
            contests.addAll(contestHappyOJ);
        }
        if (contestBaekjoon != null) {
            contests.addAll(contestBaekjoon);
        }
        if (contestLeetCode != null) {
            contests.addAll(contestLeetCode);
        }
        if (contestDmoj != null) {
            contests.addAll(contestDmoj);
        }
        if (contestNowCoder != null) {
            contests.addAll(contestNowCoder);
        }
        if (contestAcWing != null) {
            contests.addAll(contestAcWing);
        }
        if (contestEolymp != null) {
            contests.addAll(contestEolymp);
        }
        if (contestTopcoder != null) {
            contests.addAll(contestTopcoder);
        }
        if (contestCodeChef != null) {
            contests.addAll(contestCodeChef);
        }
        if (contestLuoGu != null) {
            contests.addAll(contestLuoGu);
        }
        if (contestLightOJ != null) {
            contests.addAll(contestLightOJ);
        }
        if (contestPE != null) {
            contests.addAll(contestPE);
        }
        if (contestLanQiao != null) {
            contests.addAll(contestLanQiao);
        }
        if (contestJiSuanKe != null) {
            contests.addAll(contestJiSuanKe);
        }
        if (contestToph != null) {
            contests.addAll(contestToph);
        }
        if (contestHuawei != null) {
            contests.addAll(contestHuawei);
        }
        if (contestNaukri != null) {
            contests.addAll(contestNaukri);
        }
        if (contestGreeksforgeeks != null) {
            contests.addAll(contestGreeksforgeeks);
        }
        if (contestKep != null) {
            contests.addAll(contestKep);
        }
        if (contestCodeAny != null) {
            contests.addAll(contestCodeAny);
        }
        if (contestbsuir != null) {
            contests.addAll(contestbsuir);
        }
        Collections.sort(contests.subList(0, contests.size()), new Comparator<RecentContest>() {
            public int compare(RecentContest r1, RecentContest r2) {
                Date now = new Date();
                if (r1.getStart_date().getTime() <= now.getTime() && r2.getStart_date().getTime() <= now.getTime()) {
                    long diff = r1.getEnd_date().getTime() - r2.getEnd_date().getTime();
                    if (diff > 0) {
                        return 1;
                    }else if (diff < 0) {
                        return -1;
                    }
                }
                long diff = r1.getStart_date().getTime() - r2.getStart_date().getTime();
                if (diff > 0) {
                    return 1;
                }else if (diff < 0) {
                    return -1;
                }
                return 0;
            }
        });

        map.put("createDate", new Date());
        map.put("support_list", support_list);
        map.put("contests", contests);
        ObjectMapper mapper = new ObjectMapper();
        String jsonString = null;
        try {
            jsonString = mapper.writeValueAsString(map);
        } catch (JsonProcessingException e) {
            throw new RuntimeException(e);
        }

        ApplicationHome h = new ApplicationHome(OJCollector.class);
        if (h == null || h.getSource() == null || h.getSource().getParentFile() == null) {
            return;
        }
        String dirPath1 = h.getSource().getParentFile().toString()+"/data/";
        File file = new File(dirPath1);
        if (!file.exists()) {
            if (file.mkdir()) {
            }else{
                log.error(dirPath1 + " create failed.");
            }
        }
        StreamHandler.write(dirPath1 + "otheroj.json", jsonString);
        log.info("Collect contests items " + contests.size());
    }

    public static void main(String[] args) {
        java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
        java.util.logging.Logger.getLogger("org.apache.commons.httpclient").setLevel(Level.OFF);
        CollectContests();
    }
}
